Code
suppressPackageStartupMessages(library(tidyr))
suppressPackageStartupMessages(library(psych))
suppressPackageStartupMessages(library(ggplot2))
suppressPackageStartupMessages(library(patchwork))
# Combine Y/PY and N/PN as suggested in the RoB 2 tool guide
factorize_question = function(x) factor(x, labels=c("Y", "Y", "N", "N", "NI", "NA"), levels=c("Y", "PY", "N", "PN", "NI", "NA"))
factorize_judgement = function(x) factor(x, levels=c("High", "Some concerns", "Low", "NA"))
questions = list(
"dim1" = c("1.1" = "sequence_random", "1.2" = "allocation_concealed", "1.3" = "baseline_differences"),
"dim2_assignment" = c("2.1" = "participants_aware", "2.2" = "carers_aware", "2.3_ass" = "deviations_intended_intervention", "2.4_ass" = "deviations_balanced", "2.5_ass" = "deviations_affected_outcome", "2.6" = "appropriate_analysis", "2.7_ass" = "ITT_failure"),
"dim2_adhering" = c("2.1" = "participants_aware", "2.2" = "carers_aware", "2.3_adh" = "cointerventions_balanced", "2.4_adh" = "failures_implementing_interv", "2.5_adh" = "participants_adherence", "2.6" = "appropriate_analysis"),
"dim3" = c("3.1" = "outcome_available", "3.2" = "missing_outcome_data", "3.3" = "missingness_could", "3.4" = "missing_differences", "3.5" = "missingness_likely"),
"dim4" = c("4.1" = "measuring_outcome", "4.2" = "outcome_differences", "4.3" = "assessors_aware", "4.4" = "assessment_influenced_could", "4.5" = "assessment_influenced_likely"),
"dim5" = c("5.1" = "SAP", "5.2" = "result_selected_multiple_outcome", "5.3" = "result_selected_mult_analyses")
)
all_questions = unique(unname(unlist(questions)))
judgements = c("rob_dim1", "rob_dim2", "rob_dim3", "rob_dim4", "rob_dim5", "overall")
results = read.csv2("../../data/RoB-2/Database_Minozzi_TW.csv")
results = as.data.frame(pivot_wider(results, id_cols=ID, names_from=rater, values_from=all_of(c(all_questions, judgements, "Time"))))[2:129] # remove ID (rquals rownames 1:70) and Time columns
results[is.na(results)] = "NA"
# Reproduce kappa values from paper
cohen.kappa(apply(results[,paste0(questions[["dim1"]]["1.1"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.5931794
Code
cohen.kappa(apply(results[,paste0(questions[["dim1"]]["1.2"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.5916411
Code
cohen.kappa(apply(results[,paste0(questions[["dim1"]]["1.3"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.3403349
Code
cohen.kappa(apply(results[,paste0(judgements[1], "_", 1:4)], 2, factorize_judgement))$av.kappa[1] 0.4544605
Code
assignments = which(rowSums(results[,paste0(questions[["dim2_adhering"]]["2.5_adh"], "_", 1:4)] == "NA")==4) # turns out to be 1:17
adhering = which(rowSums(results[,paste0(questions[["dim2_adhering"]]["2.5_adh"], "_", 1:4)] == "NA")!=4) # turns out to be 38:70
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.1"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.4346561
Code
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.2"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.1971975
Code
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.3_ass"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.168771
Code
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.4_ass"], "_", 1:4)], 2, factorize_question))$av.kappaWarning in cohen.kappa1(x1, w = w, n.obs = n.obs, alpha = alpha, levels =
levels): upper or lower confidence interval exceed abs(1) and set to +/- 1.
Warning in cohen.kappa1(x1, w = w, n.obs = n.obs, alpha = alpha, levels =
levels): upper or lower confidence interval exceed abs(1) and set to +/- 1.
[1] 0.2535341
Code
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.5_ass"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.1219529
Code
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.6"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.1706083
Code
cohen.kappa(apply(results[assignments, paste0(questions[["dim2_assignment"]]["2.7_ass"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.08326283
Code
cohen.kappa(apply(results[assignments, paste0(judgements[2], "_", 1:4)], 2, factorize_judgement))$av.kappa[1] 0.08650628
Code
cohen.kappa(apply(results[adhering, paste0(questions[["dim2_adhering"]]["2.1"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.6210661
Code
cohen.kappa(apply(results[adhering, paste0(questions[["dim2_adhering"]]["2.2"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.6373535
Code
cohen.kappa(apply(results[adhering, paste0(questions[["dim2_adhering"]]["2.3_adh"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.5644391
Code
cohen.kappa(apply(results[adhering, paste0(questions[["dim2_adhering"]]["2.4_adh"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.1050476
Code
cohen.kappa(apply(results[adhering, paste0(questions[["dim2_adhering"]]["2.5_adh"], "_", 1:4)], 2, factorize_question))$av.kappa # 1 in paper, but 0.19 on re-analysis?[1] 0.1914166
Code
cohen.kappa(apply(results[adhering, paste0(questions[["dim2_adhering"]]["2.6"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.1630886
Code
cohen.kappa(apply(results[adhering, paste0(judgements[2], "_", 1:4)], 2, factorize_judgement))$av.kappa[1] 0.2238221
Code
cohen.kappa(apply(results[,paste0(questions[["dim3"]]["3.1"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.3860467
Code
cohen.kappa(apply(results[,paste0(questions[["dim3"]]["3.2"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.3263355
Code
cohen.kappa(apply(results[,paste0(questions[["dim3"]]["3.3"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.2934582
Code
cohen.kappa(apply(results[,paste0(questions[["dim3"]]["3.4"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.2615411
Code
cohen.kappa(apply(results[,paste0(questions[["dim3"]]["3.5"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.1982634
Code
cohen.kappa(apply(results[,paste0(judgements[3], "_", 1:4)], 2, factorize_judgement))$av.kappa[1] 0.225426
Code
#cohen.kappa(apply(results[,paste0(questions[["dim4"]]["4.1"], "_", 1:4)], 2, factorize_question))$av.kappa # 1 in paper, NaN on re-analysis because of no variance
cohen.kappa(apply(results[,paste0(questions[["dim4"]]["4.2"], "_", 1:4)], 2, factorize_question))$av.kappa[1] -0.008476065
Code
cohen.kappa(apply(results[,paste0(questions[["dim4"]]["4.3"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.2687368
Code
cohen.kappa(apply(results[,paste0(questions[["dim4"]]["4.4"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.3919106
Code
cohen.kappa(apply(results[,paste0(questions[["dim4"]]["4.5"], "_", 1:4)], 2, factorize_question))$av.kappa[1] 0.2155404
Code
cohen.kappa(apply(results[,paste0(judgements[4], "_", 1:4)], 2, factorize_judgement))$av.kappa[1] 0.2657696
Code
cohen.kappa(apply(results[,paste0(questions[["dim5"]]["5.1"], "_", 1:4)], 2, factorize_question))$av.kappa # 0.23 in paper but 0.33 on re-analysis?[1] 0.3300674
Code
cohen.kappa(apply(results[,paste0(questions[["dim5"]]["5.2"], "_", 1:4)], 2, factorize_question))$av.kappa[1] -0.007482116
Code
cohen.kappa(apply(results[,paste0(questions[["dim5"]]["5.3"], "_", 1:4)], 2, factorize_question))$av.kappa[1] -0.0110156
Code
cohen.kappa(apply(results[,paste0(judgements[5], "_", 1:4)], 2, factorize_judgement))$av.kappa[1] 0.3183193